{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04d85a37",
   "metadata": {},
   "outputs": [],
   "source": [
    "<font color=gray size=4> 2023-03 </font> \n",
    "\n",
    "<font color=gray size=4> week02:文本图片一键式数据获取 </font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "77912cfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests_html in d:\\adobe\\lib\\site-packages (0.10.0)\n",
      "Requirement already satisfied: bs4 in d:\\adobe\\lib\\site-packages (from requests_html) (0.0.1)\n",
      "Requirement already satisfied: pyppeteer>=0.0.14 in d:\\adobe\\lib\\site-packages (from requests_html) (1.0.2)\n",
      "Requirement already satisfied: requests in d:\\adobe\\lib\\site-packages (from requests_html) (2.27.1)\n",
      "Requirement already satisfied: fake-useragent in d:\\adobe\\lib\\site-packages (from requests_html) (1.1.1)\n",
      "Requirement already satisfied: parse in d:\\adobe\\lib\\site-packages (from requests_html) (1.19.0)\n",
      "Requirement already satisfied: pyquery in d:\\adobe\\lib\\site-packages (from requests_html) (2.0.0)\n",
      "Requirement already satisfied: w3lib in d:\\adobe\\lib\\site-packages (from requests_html) (2.1.1)\n",
      "Requirement already satisfied: urllib3<2.0.0,>=1.25.8 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (1.26.8)\n",
      "Requirement already satisfied: websockets<11.0,>=10.0 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (10.4)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.42.1 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (4.65.0)\n",
      "Requirement already satisfied: appdirs<2.0.0,>=1.4.3 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (1.4.4)\n",
      "Requirement already satisfied: pyee<9.0.0,>=8.1.0 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (8.2.2)\n",
      "Requirement already satisfied: importlib-metadata>=1.4 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (6.0.0)\n",
      "Requirement already satisfied: certifi>=2021 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests_html) (2021.10.8)\n",
      "Requirement already satisfied: beautifulsoup4 in d:\\adobe\\lib\\site-packages (from bs4->requests_html) (4.11.1)\n",
      "Requirement already satisfied: lxml>=2.1 in d:\\adobe\\lib\\site-packages (from pyquery->requests_html) (4.8.0)\n",
      "Requirement already satisfied: cssselect>=1.2.0 in d:\\adobe\\lib\\site-packages (from pyquery->requests_html) (1.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in d:\\adobe\\lib\\site-packages (from requests->requests_html) (3.3)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in d:\\adobe\\lib\\site-packages (from requests->requests_html) (2.0.12)\n",
      "Requirement already satisfied: zipp>=0.5 in d:\\adobe\\lib\\site-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests_html) (3.15.0)\n",
      "Requirement already satisfied: colorama in d:\\adobe\\lib\\site-packages (from tqdm<5.0.0,>=4.42.1->pyppeteer>=0.0.14->requests_html) (0.4.4)\n",
      "Requirement already satisfied: soupsieve>1.2 in d:\\adobe\\lib\\site-packages (from beautifulsoup4->bs4->requests_html) (2.3.2.post1)\n"
     ]
    }
   ],
   "source": [
    "!pip install requests_html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "538cb366",
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession\n",
    "\n",
    "session = HTMLSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4446f6dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = session.get('https://python.org/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5b371975",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ff234dc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Connection': 'keep-alive', 'Content-Length': '50059', 'Server': 'nginx', 'Content-Type': 'text/html; charset=utf-8', 'X-Frame-Options': 'DENY', 'Via': '1.1 vegur, 1.1 varnish, 1.1 varnish', 'Accept-Ranges': 'bytes', 'Date': 'Fri, 10 Mar 2023 06:56:43 GMT', 'Age': '1177', 'X-Served-By': 'cache-iad-kiad7000025-IAD, cache-tyo11982-TYO', 'X-Cache': 'HIT, HIT', 'X-Cache-Hits': '95, 607', 'X-Timer': 'S1678431403.000371,VS0,VE0', 'Vary': 'Cookie', 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains; preload'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4301142f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag(\\'js\\', new Date()); gtag(\\'config\\', \\'G-TF35YF9CVH\\');\\nWelcome to Python.org\\n{ \"@context\": \"https://schema.org\", \"@type\": \"WebSite\", \"url\": \"https://www.python.org/\", \"potentialAction\": { \"@type\": \"SearchAction\", \"target\": \"https://www.python.org/search/?q={search_term_string}\", \"query-input\": \"required name=search_term_string\" } } var _gaq = _gaq || []; _gaq.push([\\'_setAccount\\', \\'UA-39055973-1\\']); _gaq.push([\\'_trackPageview\\']); (function() { var ga = document.createElement(\\'script\\'); ga.type = \\'text/javascript\\'; ga.async = true; ga.src = (\\'https:\\' == document.location.protocol ? \\'https://ssl\\' : \\'http://www\\') + \\'.google-analytics.com/ga.js\\'; var s = document.getElementsByTagName(\\'script\\')[0]; s.parentNode.insertBefore(ga, s); })();\\nNotice: While JavaScript is not essential for this website, your interaction with the content will be limited. Please turn JavaScript on for the full experience.\\nSkip to content\\n▼ Close\\nPython\\nPSF\\nDocs\\nPyPI\\nJobs\\nCommunity\\n▲ The Python Network\\nDonate\\n≡ Menu\\nSearch This Site GO\\nA A\\nSmaller\\nLarger\\nReset\\nSocialize\\nFacebook\\nTwitter\\nChat on IRC\\nAbout\\nApplications\\nQuotes\\nGetting Started\\nHelp\\nPython Brochure\\nDownloads\\nAll releases\\nSource code\\nWindows\\nmacOS\\nOther Platforms\\nLicense\\nAlternative Implementations\\nDocumentation\\nDocs\\nAudio/Visual Talks\\nBeginner\\'s Guide\\nDeveloper\\'s Guide\\nFAQ\\nNon-English Docs\\nPEP Index\\nPython Books\\nPython Essays\\nCommunity\\nDiversity\\nMailing Lists\\nIRC\\nForums\\nPSF Annual Impact Report\\nPython Conferences\\nSpecial Interest Groups\\nPython Logo\\nPython Wiki\\nCode of Conduct\\nCommunity Awards\\nGet Involved\\nShared Stories\\nSuccess Stories\\nArts\\nBusiness\\nEducation\\nEngineering\\nGovernment\\nScientific\\nSoftware Development\\nNews\\nPython News\\nPSF Newsletter\\nPSF News\\nPyCon US News\\nNews from the Community\\nEvents\\nPython Events\\nUser Group Events\\nPython Events Archive\\nUser Group Events Archive\\nSubmit an Event\\n>_ Launch Interactive Shell\\n# Python 3: Fibonacci series up to n >>> def fib(n): >>> a, b = 0, 1 >>> while a < n: >>> print(a, end=\\' \\') >>> a, b = b, a+b >>> print() >>> fib(1000) 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987\\nFunctions Defined\\nThe core of extensible programming is defining functions. Python allows mandatory and optional arguments, keyword arguments, and even arbitrary argument lists. More about defining functions in Python\\xa03\\n# Python 3: List comprehensions >>> fruits = [\\'Banana\\', \\'Apple\\', \\'Lime\\'] >>> loud_fruits = [fruit.upper() for fruit in fruits] >>> print(loud_fruits) [\\'BANANA\\', \\'APPLE\\', \\'LIME\\'] # List and the enumerate function >>> list(enumerate(fruits)) [(0, \\'Banana\\'), (1, \\'Apple\\'), (2, \\'Lime\\')]\\nCompound Data Types\\nLists (known as arrays in other languages) are one of the compound data types that Python understands. Lists can be indexed, sliced and manipulated with other built-in functions. More about lists in Python\\xa03\\n# Python 3: Simple arithmetic >>> 1 / 2 0.5 >>> 2 ** 3 8 >>> 17 / 3 # classic division returns a float 5.666666666666667 >>> 17 // 3 # floor division 5\\nIntuitive Interpretation\\nCalculations are simple with Python, and expression syntax is straightforward: the operators +, -, * and / work as expected; parentheses () can be used for grouping. More about simple math functions in Python\\xa03.\\n# For loop on a list >>> numbers = [2, 4, 6, 8] >>> product = 1 >>> for number in numbers: ... product = product * number ... >>> print(\\'The product is:\\', product) The product is: 384\\nAll the Flow You’d Expect\\nPython knows the usual control flow statements that other languages speak — if, for, while and range — with some of its own twists, of course. More control flow tools in Python\\xa03\\n# Simple output (with Unicode) >>> print(\"Hello, I\\'m Python!\") Hello, I\\'m Python! # Input, assignment >>> name = input(\\'What is your name?\\\\n\\') What is your name? Python >>> print(f\\'Hi, {name}.\\') Hi, Python.\\nQuick & Easy to Learn\\nExperienced programmers in any other language can pick up Python very quickly, and beginners find the clean syntax and indentation structure easy to learn. Whet your appetite with our Python\\xa03 overview.\\nPython is a programming language that lets you work quickly and integrate systems more effectively. Learn More\\nGet Started\\nWhether you\\'re new to programming or an experienced developer, it\\'s easy to learn and use Python.\\nStart with our Beginner’s Guide\\nDownload\\nPython source code and installers are available for download for all versions!\\nLatest: Python 3.11.2\\nDocs\\nDocumentation for Python\\'s standard library, along with tutorials and guides, are available online.\\ndocs.python.org\\nJobs\\nLooking for work or have a Python related position that you\\'re trying to hire for? Our relaunched community-run job board is the place to go.\\njobs.python.org\\nLatest News\\nMore\\n2023-03-08 Python 3.12.0 alpha 6 released\\n2023-02-15 The Case for a Second Developer-in-Residence for Python\\n2023-02-10 Python Packaging Strategy Discussion Summary - Part 1\\n2023-02-08 Python 3.11.2, Python 3.10.10 and 3.12.0 alpha 5 are available\\n2023-02-08 Announcing Python Software Foundation Fellow Members for Q4 2022! 🎉\\nUpcoming Events\\nMore\\n2023-03-13 Python Web Conf 2023\\n2023-03-15 The long white computing cloud - NZPUG-Auckland\\n2023-03-22 Python Meeting Düsseldorf\\n2023-04-01 PyTexas 2023\\n2023-04-07 PyCamp Spain 2023\\nSuccess Stories\\nMore\\nPython powers major aspects of Abridge’s ML lifecycle, including data annotation, research and experimentation, and ML model deployment to production.\\nAbridging clinical conversations using Python by Nimshi Venkat and Sandeep Konam\\nUse Python for…\\nMore\\nWeb Development: Django, Pyramid, Bottle, Tornado, Flask, web2py\\nGUI Development: tkInter, PyGObject, PyQt, PySide, Kivy, wxPython\\nScientific and Numeric: SciPy, Pandas, IPython\\nSoftware Development: Buildbot, Trac, Roundup\\nSystem Administration: Ansible, Salt, OpenStack, xonsh\\n>>> Python Enhancement Proposals (PEPs): The future of Python is discussed here. RSS\\n>>> Python Software Foundation\\nThe mission of the Python Software Foundation is to promote, protect, and advance the Python programming language, and to support and facilitate the growth of a diverse and international community of Python programmers. Learn more\\nBecome a Member Donate to the PSF\\n▲ Back to Top\\nAbout\\nApplications\\nQuotes\\nGetting Started\\nHelp\\nPython Brochure\\nDownloads\\nAll releases\\nSource code\\nWindows\\nmacOS\\nOther Platforms\\nLicense\\nAlternative Implementations\\nDocumentation\\nDocs\\nAudio/Visual Talks\\nBeginner\\'s Guide\\nDeveloper\\'s Guide\\nFAQ\\nNon-English Docs\\nPEP Index\\nPython Books\\nPython Essays\\nCommunity\\nDiversity\\nMailing Lists\\nIRC\\nForums\\nPSF Annual Impact Report\\nPython Conferences\\nSpecial Interest Groups\\nPython Logo\\nPython Wiki\\nCode of Conduct\\nCommunity Awards\\nGet Involved\\nShared Stories\\nSuccess Stories\\nArts\\nBusiness\\nEducation\\nEngineering\\nGovernment\\nScientific\\nSoftware Development\\nNews\\nPython News\\nPSF Newsletter\\nPSF News\\nPyCon US News\\nNews from the Community\\nEvents\\nPython Events\\nUser Group Events\\nPython Events Archive\\nUser Group Events Archive\\nSubmit an Event\\nContributing\\nDeveloper\\'s Guide\\nIssue Tracker\\npython-dev list\\nCore Mentorship\\nReport a Security Issue\\n▲ Back to Top\\nHelp & General Contact\\nDiversity Initiatives\\nSubmit Website Bug\\nStatus\\nCopyright ©2001-2023. \\xa0Python Software Foundation \\xa0Legal Statements \\xa0Privacy Policy \\xa0Powered by Heroku\\nwindow.jQuery || document.write(\\'<script src=\"/static/js/libs/jquery-1.8.2.min.js\"><\\\\/script>\\') window.jQuery || document.write(\\'<script src=\"/static/js/libs/jquery-ui-1.12.1.min.js\"><\\\\/script>\\')'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "99bc6920",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'/',\n",
       " '//docs.python.org/3/tutorial/',\n",
       " '//docs.python.org/3/tutorial/controlflow.html',\n",
       " '//docs.python.org/3/tutorial/controlflow.html#defining-functions',\n",
       " '//docs.python.org/3/tutorial/introduction.html#lists',\n",
       " '//jobs.python.org',\n",
       " '/about/',\n",
       " '/about/apps',\n",
       " '/about/apps/',\n",
       " '/about/gettingstarted/',\n",
       " '/about/help/',\n",
       " '/about/legal/',\n",
       " '/about/quotes/',\n",
       " '/blogs/',\n",
       " '/community-landing/',\n",
       " '/community/',\n",
       " '/community/awards',\n",
       " '/community/diversity/',\n",
       " '/community/forums/',\n",
       " '/community/irc/',\n",
       " '/community/lists/',\n",
       " '/community/logos/',\n",
       " '/community/sigs/',\n",
       " '/community/workshops/',\n",
       " '/dev/',\n",
       " '/dev/core-mentorship/',\n",
       " '/dev/peps/',\n",
       " '/dev/peps/peps.rss',\n",
       " '/dev/security/',\n",
       " '/doc/',\n",
       " '/doc/av',\n",
       " '/doc/essays/',\n",
       " '/download/alternatives',\n",
       " '/download/other/',\n",
       " '/downloads/',\n",
       " '/downloads/macos/',\n",
       " '/downloads/release/python-3112/',\n",
       " '/downloads/source/',\n",
       " '/downloads/windows/',\n",
       " '/events/',\n",
       " '/events/calendars/',\n",
       " '/events/python-events/',\n",
       " '/events/python-events/1436/',\n",
       " '/events/python-events/1454/',\n",
       " '/events/python-events/past/',\n",
       " '/events/python-user-group/',\n",
       " '/events/python-user-group/1431/',\n",
       " '/events/python-user-group/1435/',\n",
       " '/events/python-user-group/1458/',\n",
       " '/events/python-user-group/past/',\n",
       " '/jobs/',\n",
       " '/privacy/',\n",
       " '/psf-landing/',\n",
       " '/psf/',\n",
       " '/psf/annual-report/2021/',\n",
       " '/psf/community-stories/',\n",
       " '/psf/conduct/',\n",
       " '/psf/donations/',\n",
       " '/psf/get-involved/',\n",
       " '/psf/newsletter/',\n",
       " '/psf/sponsorship/sponsors/#heroku',\n",
       " '/shell/',\n",
       " '/success-stories/',\n",
       " '/success-stories/abridging-clinical-conversations-using-python/',\n",
       " '/success-stories/category/arts/',\n",
       " '/success-stories/category/business/',\n",
       " '/success-stories/category/education/',\n",
       " '/success-stories/category/engineering/',\n",
       " '/success-stories/category/government/',\n",
       " '/success-stories/category/scientific/',\n",
       " '/success-stories/category/software-development/',\n",
       " '/users/membership/',\n",
       " 'http://bottlepy.org',\n",
       " 'http://brochure.getpython.info/',\n",
       " 'http://buildbot.net/',\n",
       " 'http://docs.python.org/3/tutorial/introduction.html#using-python-as-a-calculator',\n",
       " 'http://flask.pocoo.org/',\n",
       " 'http://ipython.org',\n",
       " 'http://pandas.pydata.org/',\n",
       " 'http://planetpython.org/',\n",
       " 'http://pycon.blogspot.com/',\n",
       " 'http://pyfound.blogspot.com/',\n",
       " 'http://python.org/dev/peps/',\n",
       " 'http://roundup.sourceforge.net/',\n",
       " 'http://tornadoweb.org',\n",
       " 'http://trac.edgewall.org/',\n",
       " 'http://wiki.python.org/moin/Languages',\n",
       " 'http://wiki.python.org/moin/TkInter',\n",
       " 'http://www.ansible.com',\n",
       " 'http://www.djangoproject.com/',\n",
       " 'http://www.pylonsproject.org/',\n",
       " 'http://www.riverbankcomputing.co.uk/software/pyqt/intro',\n",
       " 'http://www.scipy.org',\n",
       " 'http://www.web2py.com/',\n",
       " 'http://www.wxpython.org/',\n",
       " 'https://blog.python.org',\n",
       " 'https://bugs.python.org/',\n",
       " 'https://devguide.python.org/',\n",
       " 'https://docs.python.org',\n",
       " 'https://docs.python.org/3/license.html',\n",
       " 'https://docs.python.org/faq/',\n",
       " 'https://github.com/python/pythondotorg/issues',\n",
       " 'https://kivy.org/',\n",
       " 'https://mail.python.org/mailman/listinfo/python-dev',\n",
       " 'https://psfmember.org/civicrm/contribute/transact?reset=1&id=2',\n",
       " 'https://pyfound.blogspot.com/2023/02/announcing-python-software-foundation.html',\n",
       " 'https://pyfound.blogspot.com/2023/02/python-packaging-strategy-discussion.html',\n",
       " 'https://pyfound.blogspot.com/2023/02/the-case-for-second-developer-in.html',\n",
       " 'https://pypi.org/',\n",
       " 'https://pythoninsider.blogspot.com/2023/02/python-3112-python-31010-and-3120-alpha.html',\n",
       " 'https://pythoninsider.blogspot.com/2023/03/python-3120-alpha-6-released.html',\n",
       " 'https://saltproject.io',\n",
       " 'https://status.python.org/',\n",
       " 'https://twitter.com/ThePSF',\n",
       " 'https://wiki.gnome.org/Projects/PyGObject',\n",
       " 'https://wiki.python.org/moin/',\n",
       " 'https://wiki.python.org/moin/BeginnersGuide',\n",
       " 'https://wiki.python.org/moin/PythonBooks',\n",
       " 'https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event',\n",
       " 'https://wiki.qt.io/PySide',\n",
       " 'https://www.facebook.com/pythonlang?fref=ts',\n",
       " 'https://www.openstack.org',\n",
       " 'https://www.python.org/psf/',\n",
       " 'https://xon.sh'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8e48b527",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'http://bottlepy.org',\n",
       " 'http://brochure.getpython.info/',\n",
       " 'http://buildbot.net/',\n",
       " 'http://docs.python.org/3/tutorial/introduction.html#using-python-as-a-calculator',\n",
       " 'http://flask.pocoo.org/',\n",
       " 'http://ipython.org',\n",
       " 'http://pandas.pydata.org/',\n",
       " 'http://planetpython.org/',\n",
       " 'http://pycon.blogspot.com/',\n",
       " 'http://pyfound.blogspot.com/',\n",
       " 'http://python.org/dev/peps/',\n",
       " 'http://roundup.sourceforge.net/',\n",
       " 'http://tornadoweb.org',\n",
       " 'http://trac.edgewall.org/',\n",
       " 'http://wiki.python.org/moin/Languages',\n",
       " 'http://wiki.python.org/moin/TkInter',\n",
       " 'http://www.ansible.com',\n",
       " 'http://www.djangoproject.com/',\n",
       " 'http://www.pylonsproject.org/',\n",
       " 'http://www.riverbankcomputing.co.uk/software/pyqt/intro',\n",
       " 'http://www.scipy.org',\n",
       " 'http://www.web2py.com/',\n",
       " 'http://www.wxpython.org/',\n",
       " 'https://blog.python.org',\n",
       " 'https://bugs.python.org/',\n",
       " 'https://devguide.python.org/',\n",
       " 'https://docs.python.org',\n",
       " 'https://docs.python.org/3/license.html',\n",
       " 'https://docs.python.org/3/tutorial/',\n",
       " 'https://docs.python.org/3/tutorial/controlflow.html',\n",
       " 'https://docs.python.org/3/tutorial/controlflow.html#defining-functions',\n",
       " 'https://docs.python.org/3/tutorial/introduction.html#lists',\n",
       " 'https://docs.python.org/faq/',\n",
       " 'https://github.com/python/pythondotorg/issues',\n",
       " 'https://jobs.python.org',\n",
       " 'https://kivy.org/',\n",
       " 'https://mail.python.org/mailman/listinfo/python-dev',\n",
       " 'https://psfmember.org/civicrm/contribute/transact?reset=1&id=2',\n",
       " 'https://pyfound.blogspot.com/2023/02/announcing-python-software-foundation.html',\n",
       " 'https://pyfound.blogspot.com/2023/02/python-packaging-strategy-discussion.html',\n",
       " 'https://pyfound.blogspot.com/2023/02/the-case-for-second-developer-in.html',\n",
       " 'https://pypi.org/',\n",
       " 'https://pythoninsider.blogspot.com/2023/02/python-3112-python-31010-and-3120-alpha.html',\n",
       " 'https://pythoninsider.blogspot.com/2023/03/python-3120-alpha-6-released.html',\n",
       " 'https://saltproject.io',\n",
       " 'https://status.python.org/',\n",
       " 'https://twitter.com/ThePSF',\n",
       " 'https://wiki.gnome.org/Projects/PyGObject',\n",
       " 'https://wiki.python.org/moin/',\n",
       " 'https://wiki.python.org/moin/BeginnersGuide',\n",
       " 'https://wiki.python.org/moin/PythonBooks',\n",
       " 'https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event',\n",
       " 'https://wiki.qt.io/PySide',\n",
       " 'https://www.facebook.com/pythonlang?fref=ts',\n",
       " 'https://www.openstack.org',\n",
       " 'https://www.python.org/',\n",
       " 'https://www.python.org/about/',\n",
       " 'https://www.python.org/about/apps',\n",
       " 'https://www.python.org/about/apps/',\n",
       " 'https://www.python.org/about/gettingstarted/',\n",
       " 'https://www.python.org/about/help/',\n",
       " 'https://www.python.org/about/legal/',\n",
       " 'https://www.python.org/about/quotes/',\n",
       " 'https://www.python.org/blogs/',\n",
       " 'https://www.python.org/community-landing/',\n",
       " 'https://www.python.org/community/',\n",
       " 'https://www.python.org/community/awards',\n",
       " 'https://www.python.org/community/diversity/',\n",
       " 'https://www.python.org/community/forums/',\n",
       " 'https://www.python.org/community/irc/',\n",
       " 'https://www.python.org/community/lists/',\n",
       " 'https://www.python.org/community/logos/',\n",
       " 'https://www.python.org/community/sigs/',\n",
       " 'https://www.python.org/community/workshops/',\n",
       " 'https://www.python.org/dev/',\n",
       " 'https://www.python.org/dev/core-mentorship/',\n",
       " 'https://www.python.org/dev/peps/',\n",
       " 'https://www.python.org/dev/peps/peps.rss',\n",
       " 'https://www.python.org/dev/security/',\n",
       " 'https://www.python.org/doc/',\n",
       " 'https://www.python.org/doc/av',\n",
       " 'https://www.python.org/doc/essays/',\n",
       " 'https://www.python.org/download/alternatives',\n",
       " 'https://www.python.org/download/other/',\n",
       " 'https://www.python.org/downloads/',\n",
       " 'https://www.python.org/downloads/macos/',\n",
       " 'https://www.python.org/downloads/release/python-3112/',\n",
       " 'https://www.python.org/downloads/source/',\n",
       " 'https://www.python.org/downloads/windows/',\n",
       " 'https://www.python.org/events/',\n",
       " 'https://www.python.org/events/calendars/',\n",
       " 'https://www.python.org/events/python-events/',\n",
       " 'https://www.python.org/events/python-events/1436/',\n",
       " 'https://www.python.org/events/python-events/1454/',\n",
       " 'https://www.python.org/events/python-events/past/',\n",
       " 'https://www.python.org/events/python-user-group/',\n",
       " 'https://www.python.org/events/python-user-group/1431/',\n",
       " 'https://www.python.org/events/python-user-group/1435/',\n",
       " 'https://www.python.org/events/python-user-group/1458/',\n",
       " 'https://www.python.org/events/python-user-group/past/',\n",
       " 'https://www.python.org/jobs/',\n",
       " 'https://www.python.org/privacy/',\n",
       " 'https://www.python.org/psf-landing/',\n",
       " 'https://www.python.org/psf/',\n",
       " 'https://www.python.org/psf/annual-report/2021/',\n",
       " 'https://www.python.org/psf/community-stories/',\n",
       " 'https://www.python.org/psf/conduct/',\n",
       " 'https://www.python.org/psf/donations/',\n",
       " 'https://www.python.org/psf/get-involved/',\n",
       " 'https://www.python.org/psf/newsletter/',\n",
       " 'https://www.python.org/psf/sponsorship/sponsors/#heroku',\n",
       " 'https://www.python.org/shell/',\n",
       " 'https://www.python.org/success-stories/',\n",
       " 'https://www.python.org/success-stories/abridging-clinical-conversations-using-python/',\n",
       " 'https://www.python.org/success-stories/category/arts/',\n",
       " 'https://www.python.org/success-stories/category/business/',\n",
       " 'https://www.python.org/success-stories/category/education/',\n",
       " 'https://www.python.org/success-stories/category/engineering/',\n",
       " 'https://www.python.org/success-stories/category/government/',\n",
       " 'https://www.python.org/success-stories/category/scientific/',\n",
       " 'https://www.python.org/success-stories/category/software-development/',\n",
       " 'https://www.python.org/users/membership/',\n",
       " 'https://xon.sh'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.absolute_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "18841594",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "About\n",
      "Applications\n",
      "Quotes\n",
      "Getting Started\n",
      "Help\n",
      "Python Brochure\n"
     ]
    }
   ],
   "source": [
    "about = r.html.find('#about', first=True)\n",
    "print(about.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "959fcbcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'about', 'class': ('tier-1', 'element-1'), 'aria-haspopup': 'true'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.attrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "81ce95be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<li id=\"about\" class=\"tier-1 element-1\" aria-haspopup=\"true\">\\n<a href=\"/about/\" title=\"\" class=\"\">About</a>\\n<ul class=\"subnav menu\" role=\"menu\" aria-hidden=\"true\">\\n<li class=\"tier-2 element-1\" role=\"treeitem\"><a href=\"/about/apps/\" title=\"\">Applications</a></li>\\n<li class=\"tier-2 element-2\" role=\"treeitem\"><a href=\"/about/quotes/\" title=\"\">Quotes</a></li>\\n<li class=\"tier-2 element-3\" role=\"treeitem\"><a href=\"/about/gettingstarted/\" title=\"\">Getting Started</a></li>\\n<li class=\"tier-2 element-4\" role=\"treeitem\"><a href=\"/about/help/\" title=\"\">Help</a></li>\\n<li class=\"tier-2 element-5\" role=\"treeitem\"><a href=\"http://brochure.getpython.info/\" title=\"\">Python Brochure</a></li>\\n</ul>\\n</li>'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5d63ed0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'li'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "292009f3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Element 'a' href='/about/' title='' class=()>,\n",
       " <Element 'a' href='/about/apps/' title=''>,\n",
       " <Element 'a' href='/about/quotes/' title=''>,\n",
       " <Element 'a' href='/about/gettingstarted/' title=''>,\n",
       " <Element 'a' href='/about/help/' title=''>,\n",
       " <Element 'a' href='http://brochure.getpython.info/' title=''>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.find('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8ca37760",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OpenAI大幅降价ChatGPT：或导致人工智能行业走向垄断\n",
      "{'https://news.cnblogs.com/n/738002/'}\n",
      "ChatGPT黑化版来了！拉踩用户智商、写暴力故事，bug只因3个字母\n",
      "{'https://news.cnblogs.com/n/737949/'}\n",
      "Nature凌晨发稿：物理层巨大突破 21℃实现室温超导\n",
      "{'https://news.cnblogs.com/n/737907/'}\n",
      "ChatGPT版机械佛Hotoke AI爆火出圈，上线5天解决13000+烦恼\n",
      "{'https://news.cnblogs.com/n/737874/'}\n",
      "一人改代码搞崩推特，马斯克气疯：全部重写！\n",
      "{'https://news.cnblogs.com/n/737816/'}\n",
      "大神李沐被曝离职！投身大模型创业，GitHub项目已开\n",
      "{'https://news.cnblogs.com/n/737809/'}\n",
      "微软杀疯了！几行代码创建私人定制ChatGPT，AI办公软件帝国来了\n",
      "{'https://news.cnblogs.com/n/737793/'}\n",
      "便宜顺畅、中文友好……自打用上Notion AI，ChatGPT都不香了！\n",
      "{'https://news.cnblogs.com/n/737780/'}\n",
      "“AI读脑术”：大脑视觉信号被Stable Diffusion复现图像\n",
      "{'https://news.cnblogs.com/n/737715/'}\n",
      "日本国土交通部在Unity商店发布“全日本城市3D模型”，全部免费用！\n",
      "{'https://news.cnblogs.com/n/737610/'}\n",
      "aardio 编程语言作者：因妻子患癌，再无精力维护项目\n",
      "{'https://news.cnblogs.com/n/737591/'}\n",
      "阿里平头哥披露RISC-V商业生态全景，已完成主流操作系统全适配\n",
      "{'https://news.cnblogs.com/n/737548/'}\n",
      "Visual Studio 2022 17.6 发布首个预览版\n",
      "{'https://news.cnblogs.com/n/737538/'}\n",
      "ChatGPT官方API发布：输出100万词只要2.7美金，成本砍掉90%\n",
      "{'https://news.cnblogs.com/n/737522/'}\n",
      "Google 面对微软的 ChatGPT 之战，可能注定失败\n",
      "{'https://news.cnblogs.com/n/737449/'}\n",
      "未来计算机在脑细胞上运行？“类器官智能”计划和路线图公布\n",
      "{'https://news.cnblogs.com/n/737432/'}\n",
      "用量子理论凭空生成能量：物理学家学会使用零点能\n",
      "{'https://news.cnblogs.com/n/737431/'}\n",
      "小扎、马斯克宣战ChatGPT！Meta和推特组建顶级AI团队，硅谷硝烟四起\n",
      "{'https://news.cnblogs.com/n/737394/'}\n"
     ]
    }
   ],
   "source": [
    "r = session.get(\"https://news.cnblogs.com/n/recommend\")\n",
    "\n",
    "# 通过CSS找到新闻标签\n",
    "\n",
    "sel = 'h2.news_entry > a'\n",
    "news = r.html.find(sel) # print (new[1].text) 没有循环 只找一个结果\n",
    "for new in news:\n",
    "    print(new.text)\n",
    "    print(new.absolute_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "13bfc030",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OpenAI大幅降价ChatGPT：或导致人工智能行业走向垄断\n",
      "北京时间 3 月 10 日早间消息，据报道，为当前人工智能热潮提供支撑的最重要“原材料”的价格正在快速下降，这将有助于这项技术更快地进入主流。然而，这也威胁到了那些希望从这股热潮中获利的创业公司的财务状况，并可能导致行业主导权集中在一小部分公司的手中。 这里的原材料指的是大语言模型（LLM）的处理能\n",
      "itwriter 投递  评论(0) 332 人浏览 OpenAI ChatGPT 发布于 2023-03-10 09:18\n",
      "{'https://news.cnblogs.com/n/topic_859.htm', 'https://news.cnblogs.com/n/738002/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/738002#comment', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/tag/OpenAI/'}\n",
      "ChatGPT黑化版来了！拉踩用户智商、写暴力故事，bug只因3个字母\n",
      "杨净羿阁发自凹非寺 量子位公众号 QbitAI ChatGPT“越狱”后，彻底放飞自我了。 直接无视掉 OpenAI 给它在安全、伦理上限定的条条框框—— 写暴力故事、拉踩用户智商、随意预测未来……简直无恶不作！ 而这一切背后竟然只靠一位 Reddit 用户实现，他自创了一套给 ChatGPT“洗脑\n",
      "itwriter 投递  评论(0) 870 人浏览 ChatGPT 发布于 2023-03-09 13:23\n",
      "{'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/737949#comment', 'https://news.cnblogs.com/n/topic_2945.htm', 'https://news.cnblogs.com/n/737949/'}\n",
      "Nature凌晨发稿：物理层巨大突破 21℃实现室温超导\n",
      "鱼羊萧箫发自凹非寺 量子位公众号 QbitAI 物理学界又被扔下一枚核弹！ 还是因为那石破天惊、看上去分分钟要把诺奖斩获马下的四个字： 室温超导。 并且这次，来自罗彻斯特大学的 Ranga Dias 团队，给出的结果压强更低，临界温度更高：新材料在约 21℃的室温条件下，加压到 1 万个标准大气压就\n",
      "itwriter 投递  评论(0) 1306 人浏览 室温超导 发布于 2023-03-09 08:12\n",
      "{'https://news.cnblogs.com/n/tag/%E5%AE%A4%E6%B8%A9%E8%B6%85%E5%AF%BC/', 'https://news.cnblogs.com/n/737907/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737907#comment', 'https://news.cnblogs.com/n/topic_319.htm'}\n",
      "ChatGPT版机械佛Hotoke AI爆火出圈，上线5天解决13000+烦恼\n",
      "白交发自凹非寺 量子位公众号 QbitAI ChatGPT 现在已经在为人类“佛系”排忧解难了。 这不，日本一款 ChatGPT 版机械佛 Hotoke AI（hotoke 是日语里的佛）爆火出圈，上线 5 天已有 13000+ 烦恼被解决。 在官方界面上也赫然写着：AI 永远在你身边。 人类随便问\n",
      "itwriter 投递  评论(0) 1582 人浏览 ChatGPT 发布于 2023-03-08 12:47\n",
      "{'https://news.cnblogs.com/n/737874#comment', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/topic_2945.htm', 'https://news.cnblogs.com/n/737874/'}\n",
      "一人改代码搞崩推特，马斯克气疯：全部重写！\n",
      "鱼羊丰色发自凹非寺 量子位公众号 QbitAI “马斯克气疯了”！就在今天，推特又双叒崩溃了，图片和任何链接都打不开。据马斯克自己的说法，原因只是“API 的一个小改动”。但后果很严重，起码在网友们看来，推特的现场是酱婶的： 而吃瓜群众们的心情，则是： 并非网友们太过刻薄，只是这事儿吧，确实有点孽力\n",
      "itwriter 投递  评论(0) 2527 人浏览 Twitter 马斯克 发布于 2023-03-07 17:34\n",
      "{'https://news.cnblogs.com/n/737816/', 'https://news.cnblogs.com/n/topic_111.htm', 'https://news.cnblogs.com/n/737816#comment', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/tag/%E9%A9%AC%E6%96%AF%E5%85%8B/', 'https://news.cnblogs.com/n/tag/Twitter/'}\n",
      "大神李沐被曝离职！投身大模型创业，GitHub项目已开\n",
      "梦晨鱼羊发自凹非寺 量子位公众号 QbitAI 亚马逊首席科学家李沐，被曝已离职创业！ 据公众号“亲爱的数据”消息，其创业项目，名为 Boson.ai。 从官网信息来看，创业内容与大模型应用有关。 与其携手创业的，是李沐的导师，另一位前亚马逊 AI 大牛：Alex Smola。 领英资料显示，这位前\n",
      "itwriter 投递  评论(0) 1845 人浏览 李沐 发布于 2023-03-07 14:28\n",
      "{'https://news.cnblogs.com/n/topic_335.htm', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737809#comment', 'https://news.cnblogs.com/n/737809/', 'https://news.cnblogs.com/n/tag/%E6%9D%8E%E6%B2%90/'}\n",
      "微软杀疯了！几行代码创建私人定制ChatGPT，AI办公软件帝国来了\n",
      "新智元报道 编辑：编辑部 微软又用 ChatGPT 出逆天操作了！Power Virtual Agents 和 AI Builder 推出了由 Azure OpenAI 服务支持的下一代 AI 功能。低代码技术，在彻底改变传统的开发格局。 微软真是逆天了。眼看，它就要用 ChatGPT 建起一个世最\n",
      "itwriter 投递  评论(0) 866 人浏览 ChatGPT 微软 发布于 2023-03-07 12:00\n",
      "{'https://news.cnblogs.com/n/tag/%E5%BE%AE%E8%BD%AF/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/topic_2.htm', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/737793#comment', 'https://news.cnblogs.com/n/737793/'}\n",
      "便宜顺畅、中文友好……自打用上Notion AI，ChatGPT都不香了！\n",
      "文 Juny 编辑 VickyXiao 去年 11 月底，当 ChatGPT 横空出世、惊艳亮相的时候，很多人可能并不知道，当时在硅谷另一个角落，也悄悄上线了其 AI 服务，并向用户开放了预约。 最近几个月，当市场的注意力几乎全被 ChatGPT、New Bing 以及科技巨头们的 AI 大动作而吸\n",
      "itwriter 投递  评论(0) 895 人浏览 Notion 发布于 2023-03-07 10:31\n",
      "{'https://news.cnblogs.com/n/737780/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/topic_2960.htm', 'https://news.cnblogs.com/n/tag/Notion/', 'https://news.cnblogs.com/n/737780#comment'}\n",
      "“AI读脑术”：大脑视觉信号被Stable Diffusion复现图像\n",
      "原标题：大脑视觉信号被 Stable Diffusion 复现图像！日本新研究炸了 丰色萧箫发自凹非寺 量子位公众号 QbitAI “现在 Stable Diffusion 已经能重建大脑视觉信号了！” 就在昨晚，一个听起来细思极恐的“AI 读脑术”研究，在网上掀起轩然大波： 这项研究声称，只需用\n",
      "itwriter 投递  评论(0) 597 人浏览 Stable Diffusion 发布于 2023-03-06 10:40\n",
      "{'https://news.cnblogs.com/n/topic_2919.htm', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737715#comment', 'https://news.cnblogs.com/n/737715/', 'https://news.cnblogs.com/n/tag/Stable%20Diffusion/'}\n",
      "日本国土交通部在Unity商店发布“全日本城市3D模型”，全部免费用！\n",
      "GameLook 报道/自 2020 年起，日本国土交通部启动了 Project PLATEAU 计划，以 3D 模型的方式（文末附下载链接）为日本各大都市创建了一比一的数字孪生。截至目前，日本政府已经共计完成了 56 座城市完整的数字化 3D 模型建设，并在近期上线了专属 Unity 开发 SDK\n",
      "itwriter 投递  评论(0) 2129 人浏览 Unity 发布于 2023-03-03 12:20\n",
      "{'https://news.cnblogs.com/n/tag/Unity/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737610#comment', 'https://news.cnblogs.com/n/topic_631.htm', 'https://news.cnblogs.com/n/737610/'}\n",
      "aardio 编程语言作者：因妻子患癌，再无精力维护项目\n",
      "aardio 是一门专注于桌面软件开发的编程语言，特点是小、轻、快，体积仅 6.5MB，学习和使用成本极低。aardio 虽然小，但提供了惊人数量的开源标准库、扩展库 —— 这些库基本都是由纯 aardio 代码实现。 据介绍，aardio 的所有库基本都是由作者一个人编写，并且 17 年一直保持非\n",
      "itwriter 投递  评论(0) 2109 人浏览 aardio 发布于 2023-03-03 09:58\n",
      "{'https://news.cnblogs.com/n/tag/aardio/', 'https://news.cnblogs.com/n/topic_335.htm', 'https://news.cnblogs.com/n/737591/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737591#comment'}\n",
      "阿里平头哥披露RISC-V商业生态全景，已完成主流操作系统全适配\n",
      "芯东西（公众号：aichip001） 作者 ZeR0 编辑 漠影 芯东西 3 月 2 日报道，由阿里巴巴平头哥举办的首届玄铁 RISC-V 生态大会在上海举行。大会以“开放、连接”为主题，汇聚英特尔、谷歌、Canonical、Imagination、海尔、支付宝、网易有道、创维酷开等全球数百家企业及\n",
      "itwriter 投递  评论(0) 1082 人浏览 平头哥 发布于 2023-03-02 15:23\n",
      "{'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737548/', 'https://news.cnblogs.com/n/tag/%E5%B9%B3%E5%A4%B4%E5%93%A5/', 'https://news.cnblogs.com/n/737548#comment', 'https://news.cnblogs.com/n/topic_6.htm'}\n",
      "Visual Studio 2022 17.6 发布首个预览版\n",
      "Visual Studio 2022 17.6 的首个预览版现已发布！此版本扩展了最近发布的 17.5 版 Visual Studio，旨在提高您的工作效率，无论您是 .NET 开发人员、游戏开发人员和 / 或 C++ 开发人员。 新功能概览： IDE 生产力 括号对着色 Git 行取消暂存 Git\n",
      "itwriter 投递  评论(0) 1417 人浏览 Visual Studio 发布于 2023-03-02 13:35\n",
      "{'https://news.cnblogs.com/n/737538/', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737538#comment', 'https://news.cnblogs.com/n/tag/Visual%20Studio/', 'https://news.cnblogs.com/n/topic_203.htm'}\n",
      "ChatGPT官方API发布：输出100万词只要2.7美金，成本砍掉90%\n",
      "智东西（公众号：zhidxcom） 编译程茜 编辑心缘 智东西 3 月 2 日报道，今日凌晨，OpenAI 放出了真正的 ChatGPT API，不是背后的 GPT-3.5 大模型，是 ChatGPT 的本体模型！ChatGPT API 价格为 1k tokens/$0.002，等于每输出 100\n",
      "itwriter 投递  评论(0) 1068 人浏览 ChatGPT OpenAI 发布于 2023-03-02 11:10\n",
      "{'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/737522#comment', 'https://news.cnblogs.com/n/topic_2945.htm', 'https://news.cnblogs.com/n/737522/', 'https://news.cnblogs.com/n/tag/OpenAI/'}\n",
      "Google 面对微软的 ChatGPT 之战，可能注定失败\n",
      "「万年老二」的快乐，微软懂得。 以往，第二都是那个不会被记住的角色。第二个登录月球的宇航员，第二个做出灯泡的发明家哦，第二个市值破万亿美元的公司……它们不是不出色，但有阿姆斯特朗、爱迪生、苹果公司珠玉在前，第二名总是更难被记住。 只是在名声没那么大、赚得没那么多之外，第二名也有第一名羡慕的机会。 第\n",
      "itwriter 投递  评论(0) 1152 人浏览 Google ChatGPT 发布于 2023-03-01 11:20\n",
      "{'https://news.cnblogs.com/n/tag/Google/', 'https://news.cnblogs.com/n/topic_1.htm', 'https://news.cnblogs.com/n/737449#comment', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/737449/'}\n",
      "未来计算机在脑细胞上运行？“类器官智能”计划和路线图公布\n",
      "人工智能 (AI) 长期以来一直受到人类大脑结构和运作方式等方面的启发，并且人工智能已经取得了从辅助诊断医疗到创作诗歌等一系列令人印象深刻的成就。尽管如此，原始模型——人类大脑——仍然在许多方面仍然优于机器。 2 月 28 日，美国约翰斯·霍普金斯大学（Johns Hopkins Universit\n",
      "itwriter 投递  评论(0) 480 人浏览 发布于 2023-03-01 09:10\n",
      "{'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737432#comment', 'https://news.cnblogs.com/n/737432/', 'https://news.cnblogs.com/n/topic_335.htm'}\n",
      "用量子理论凭空生成能量：物理学家学会使用零点能\n",
      "选自量子杂志 作者：Charlie Wood 机器之心编译 编辑：泽南、陈萍 直接提取真空零点能是在造永动机，违背能量守恒定律，现在有人通过从远处「借用」的方法实现了突破。 这是一件人们无法想象的事，物理学家已经实现用量子等价物从接近真空中凭空变出能量，这一壮举似乎违背了物理定律和常识。 「你不能直\n",
      "itwriter 投递  评论(0) 185 人浏览 量子计算 发布于 2023-03-01 09:00\n",
      "{'https://news.cnblogs.com/n/topic_1259.htm', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737431/', 'https://news.cnblogs.com/n/737431#comment', 'https://news.cnblogs.com/n/tag/%E9%87%8F%E5%AD%90%E8%AE%A1%E7%AE%97/'}\n",
      "小扎、马斯克宣战ChatGPT！Meta和推特组建顶级AI团队，硅谷硝烟四起\n",
      "新智元报道 编辑：编辑部 ChatGPT 大厂混战，怎能少得了 Meta 和推特？近日，小扎和马斯克也官宣下场，要做自己的 ChatGPT 了。 ChatGPT 的爆火，直接改变了整个硅谷大厂的格局。 微软的步步出招，倒逼谷歌走出固步自封的境地，紧急发布 AI 聊天机器人 Bard。 而硅谷大厂中的\n",
      "itwriter 投递  评论(0) 706 人浏览 ChatGPT Meta Twitter 发布于 2023-02-28 13:50\n",
      "{'https://news.cnblogs.com/n/topic_2801.htm', 'https://home.cnblogs.com/u/itwriter/', 'https://news.cnblogs.com/n/737394/', 'https://news.cnblogs.com/n/tag/ChatGPT/', 'https://news.cnblogs.com/n/tag/Twitter/', 'https://news.cnblogs.com/n/737394#comment', 'https://news.cnblogs.com/n/tag/Meta/'}\n"
     ]
    }
   ],
   "source": [
    "r = session.get(\"https://news.cnblogs.com/n/recommend\")\n",
    "\n",
    "# 通过CSS找到新闻标签\n",
    "\n",
    "sel = 'div.content'\n",
    "news = r.html.find(sel) # print (new[1].text) 没有循环 只找一个结果\n",
    "for new in news:\n",
    "    print(new.text)\n",
    "    print(new.absolute_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e9d159b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 练习：翻页"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9a2a5843",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n"
     ]
    }
   ],
   "source": [
    "url_队列 = []\n",
    "for i in range(100): # 循环\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6bf0338f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://news.cnblogs.com/n/recommend?page=1\n",
      "https://news.cnblogs.com/n/recommend?page=2\n",
      "https://news.cnblogs.com/n/recommend?page=3\n",
      "https://news.cnblogs.com/n/recommend?page=4\n",
      "https://news.cnblogs.com/n/recommend?page=5\n",
      "https://news.cnblogs.com/n/recommend?page=6\n",
      "https://news.cnblogs.com/n/recommend?page=7\n",
      "https://news.cnblogs.com/n/recommend?page=8\n",
      "https://news.cnblogs.com/n/recommend?page=9\n",
      "https://news.cnblogs.com/n/recommend?page=10\n",
      "https://news.cnblogs.com/n/recommend?page=11\n",
      "https://news.cnblogs.com/n/recommend?page=12\n",
      "https://news.cnblogs.com/n/recommend?page=13\n",
      "https://news.cnblogs.com/n/recommend?page=14\n",
      "https://news.cnblogs.com/n/recommend?page=15\n",
      "https://news.cnblogs.com/n/recommend?page=16\n",
      "https://news.cnblogs.com/n/recommend?page=17\n",
      "https://news.cnblogs.com/n/recommend?page=18\n",
      "https://news.cnblogs.com/n/recommend?page=19\n",
      "https://news.cnblogs.com/n/recommend?page=20\n",
      "https://news.cnblogs.com/n/recommend?page=21\n",
      "https://news.cnblogs.com/n/recommend?page=22\n",
      "https://news.cnblogs.com/n/recommend?page=23\n",
      "https://news.cnblogs.com/n/recommend?page=24\n",
      "https://news.cnblogs.com/n/recommend?page=25\n",
      "https://news.cnblogs.com/n/recommend?page=26\n",
      "https://news.cnblogs.com/n/recommend?page=27\n",
      "https://news.cnblogs.com/n/recommend?page=28\n",
      "https://news.cnblogs.com/n/recommend?page=29\n",
      "https://news.cnblogs.com/n/recommend?page=30\n",
      "https://news.cnblogs.com/n/recommend?page=31\n",
      "https://news.cnblogs.com/n/recommend?page=32\n",
      "https://news.cnblogs.com/n/recommend?page=33\n",
      "https://news.cnblogs.com/n/recommend?page=34\n",
      "https://news.cnblogs.com/n/recommend?page=35\n",
      "https://news.cnblogs.com/n/recommend?page=36\n",
      "https://news.cnblogs.com/n/recommend?page=37\n",
      "https://news.cnblogs.com/n/recommend?page=38\n",
      "https://news.cnblogs.com/n/recommend?page=39\n",
      "https://news.cnblogs.com/n/recommend?page=40\n",
      "https://news.cnblogs.com/n/recommend?page=41\n",
      "https://news.cnblogs.com/n/recommend?page=42\n",
      "https://news.cnblogs.com/n/recommend?page=43\n",
      "https://news.cnblogs.com/n/recommend?page=44\n",
      "https://news.cnblogs.com/n/recommend?page=45\n",
      "https://news.cnblogs.com/n/recommend?page=46\n",
      "https://news.cnblogs.com/n/recommend?page=47\n",
      "https://news.cnblogs.com/n/recommend?page=48\n",
      "https://news.cnblogs.com/n/recommend?page=49\n",
      "https://news.cnblogs.com/n/recommend?page=50\n",
      "https://news.cnblogs.com/n/recommend?page=51\n",
      "https://news.cnblogs.com/n/recommend?page=52\n",
      "https://news.cnblogs.com/n/recommend?page=53\n",
      "https://news.cnblogs.com/n/recommend?page=54\n",
      "https://news.cnblogs.com/n/recommend?page=55\n",
      "https://news.cnblogs.com/n/recommend?page=56\n",
      "https://news.cnblogs.com/n/recommend?page=57\n",
      "https://news.cnblogs.com/n/recommend?page=58\n",
      "https://news.cnblogs.com/n/recommend?page=59\n",
      "https://news.cnblogs.com/n/recommend?page=60\n",
      "https://news.cnblogs.com/n/recommend?page=61\n",
      "https://news.cnblogs.com/n/recommend?page=62\n",
      "https://news.cnblogs.com/n/recommend?page=63\n",
      "https://news.cnblogs.com/n/recommend?page=64\n",
      "https://news.cnblogs.com/n/recommend?page=65\n",
      "https://news.cnblogs.com/n/recommend?page=66\n",
      "https://news.cnblogs.com/n/recommend?page=67\n",
      "https://news.cnblogs.com/n/recommend?page=68\n",
      "https://news.cnblogs.com/n/recommend?page=69\n",
      "https://news.cnblogs.com/n/recommend?page=70\n",
      "https://news.cnblogs.com/n/recommend?page=71\n",
      "https://news.cnblogs.com/n/recommend?page=72\n",
      "https://news.cnblogs.com/n/recommend?page=73\n",
      "https://news.cnblogs.com/n/recommend?page=74\n",
      "https://news.cnblogs.com/n/recommend?page=75\n",
      "https://news.cnblogs.com/n/recommend?page=76\n",
      "https://news.cnblogs.com/n/recommend?page=77\n",
      "https://news.cnblogs.com/n/recommend?page=78\n",
      "https://news.cnblogs.com/n/recommend?page=79\n",
      "https://news.cnblogs.com/n/recommend?page=80\n",
      "https://news.cnblogs.com/n/recommend?page=81\n",
      "https://news.cnblogs.com/n/recommend?page=82\n",
      "https://news.cnblogs.com/n/recommend?page=83\n",
      "https://news.cnblogs.com/n/recommend?page=84\n",
      "https://news.cnblogs.com/n/recommend?page=85\n",
      "https://news.cnblogs.com/n/recommend?page=86\n",
      "https://news.cnblogs.com/n/recommend?page=87\n",
      "https://news.cnblogs.com/n/recommend?page=88\n",
      "https://news.cnblogs.com/n/recommend?page=89\n",
      "https://news.cnblogs.com/n/recommend?page=90\n",
      "https://news.cnblogs.com/n/recommend?page=91\n",
      "https://news.cnblogs.com/n/recommend?page=92\n",
      "https://news.cnblogs.com/n/recommend?page=93\n",
      "https://news.cnblogs.com/n/recommend?page=94\n",
      "https://news.cnblogs.com/n/recommend?page=95\n",
      "https://news.cnblogs.com/n/recommend?page=96\n",
      "https://news.cnblogs.com/n/recommend?page=97\n",
      "https://news.cnblogs.com/n/recommend?page=98\n",
      "https://news.cnblogs.com/n/recommend?page=99\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_队列 = []\n",
    "for i in range(1,100):\n",
    "    print('https://news.cnblogs.com/n/recommend?page='+str(i))\n",
    "url_队列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6841b1f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in url_队列:\n",
    "    r = session.get(i)\n",
    "\n",
    "    # 通过CSS找到新闻标签\n",
    "\n",
    "    sel = 'div.content'\n",
    "    news = r.html.find(sel) \n",
    "    print(\"-----------博客园新闻_第{page}页----------\".format(page=i.split('=')[1]))\n",
    "    for new in news:\n",
    "        print()\n",
    "        print(new.text)\n",
    "        print(new.absolute_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5e8bc60",
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession #会话框\n",
    "session = HTMLSession()\n",
    "\n",
    "r = session.get('https://www.nfu.edu.cn/xxyw/index.htm')\n",
    "\n",
    "# 获取页面上的所有链接。\n",
    "all_links =  r.html.links\n",
    "print(all_links)\n",
    "print()\n",
    "\n",
    "# 获取页面上的所有链接，以绝对路径的方式。\n",
    "all_absolute_links = r.html.absolute_links\n",
    "print(all_absolute_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d89cdca5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests-html in d:\\adobe\\lib\\site-packages (0.10.0)\n",
      "Requirement already satisfied: pyquery in d:\\adobe\\lib\\site-packages (from requests-html) (2.0.0)\n",
      "Requirement already satisfied: requests in d:\\adobe\\lib\\site-packages (from requests-html) (2.27.1)\n",
      "Requirement already satisfied: fake-useragent in d:\\adobe\\lib\\site-packages (from requests-html) (1.1.1)\n",
      "Requirement already satisfied: w3lib in d:\\adobe\\lib\\site-packages (from requests-html) (2.1.1)\n",
      "Requirement already satisfied: bs4 in d:\\adobe\\lib\\site-packages (from requests-html) (0.0.1)\n",
      "Requirement already satisfied: pyppeteer>=0.0.14 in d:\\adobe\\lib\\site-packages (from requests-html) (1.0.2)\n",
      "Requirement already satisfied: parse in d:\\adobe\\lib\\site-packages (from requests-html) (1.19.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.42.1 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (4.65.0)\n",
      "Requirement already satisfied: pyee<9.0.0,>=8.1.0 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (8.2.2)\n",
      "Requirement already satisfied: importlib-metadata>=1.4 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (6.0.0)\n",
      "Requirement already satisfied: websockets<11.0,>=10.0 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (10.4)\n",
      "Requirement already satisfied: certifi>=2021 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (2021.10.8)\n",
      "Requirement already satisfied: urllib3<2.0.0,>=1.25.8 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (1.26.8)\n",
      "Requirement already satisfied: appdirs<2.0.0,>=1.4.3 in d:\\adobe\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (1.4.4)\n",
      "Requirement already satisfied: beautifulsoup4 in d:\\adobe\\lib\\site-packages (from bs4->requests-html) (4.11.1)\n",
      "Requirement already satisfied: lxml>=2.1 in d:\\adobe\\lib\\site-packages (from pyquery->requests-html) (4.8.0)\n",
      "Requirement already satisfied: cssselect>=1.2.0 in d:\\adobe\\lib\\site-packages (from pyquery->requests-html) (1.2.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in d:\\adobe\\lib\\site-packages (from requests->requests-html) (3.3)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in d:\\adobe\\lib\\site-packages (from requests->requests-html) (2.0.12)\n",
      "Requirement already satisfied: zipp>=0.5 in d:\\adobe\\lib\\site-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests-html) (3.15.0)\n",
      "Requirement already satisfied: colorama in d:\\adobe\\lib\\site-packages (from tqdm<5.0.0,>=4.42.1->pyppeteer>=0.0.14->requests-html) (0.4.4)\n",
      "Requirement already satisfied: soupsieve>1.2 in d:\\adobe\\lib\\site-packages (from beautifulsoup4->bs4->requests-html) (2.3.2.post1)\n"
     ]
    }
   ],
   "source": [
    "!pip install requests-html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04cdda9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession #会话框\n",
    "session = HTMLSession()\n",
    "r = session.get('https://www.nfu.edu.cn/xxyw/index.htm')\n",
    "r."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7586475",
   "metadata": {},
   "outputs": [],
   "source": [
    "items_img = r.html.find('span.sm-thumb > a >img')\n",
    "print(items_img)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0b3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
